Skip to content

Commit 7962e22

Browse files
PatrickRenPatrickRen
authored
[fs] Upgrade filesystem Hadoop version to 3.4.2 (#6290)
1 parent 91fae29 commit 7962e22

23 files changed

Lines changed: 255 additions & 3681 deletions

File tree

‎paimon-common/src/main/java/org/apache/paimon/fs/MultiPartUploadStore.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,13 @@ C completeMultipartUpload(
4343
String objectName, String uploadId, List<T> partETags, long numBytesInParts)
4444
throws IOException;
4545

46-
T uploadPart(String objectName, String uploadId, int partNumber, File file, long byteLength)
46+
T uploadPart(
47+
String objectName,
48+
String uploadId,
49+
int partNumber,
50+
boolean isLastPart,
51+
File file,
52+
long byteLength)
4753
throws IOException;
4854

4955
void abortMultipartUpload(String objectName, String uploadId) throws IOException;

‎paimon-common/src/main/java/org/apache/paimon/fs/MultiPartUploadTwoPhaseOutputStream.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ public void write(int b) throws IOException {
7171
buffer.write(b);
7272
position++;
7373
if (buffer.size() >= partSizeThreshold()) {
74-
uploadPart();
74+
uploadPart(false);
7575
}
7676
}
7777

@@ -88,7 +88,7 @@ public void write(byte[] b, int off, int len) throws IOException {
8888
buffer.write(b, off, len);
8989
position += len;
9090
if (buffer.size() >= partSizeThreshold()) {
91-
uploadPart();
91+
uploadPart(false);
9292
}
9393
}
9494

@@ -115,14 +115,14 @@ public Committer closeForCommit() throws IOException {
115115
closed = true;
116116

117117
if (buffer.size() > 0) {
118-
uploadPart();
118+
uploadPart(true);
119119
}
120120

121121
return new MultiPartUploadCommitter(
122122
multiPartUploadStore, uploadId, uploadedParts, objectName, position);
123123
}
124124

125-
private void uploadPart() throws IOException {
125+
private void uploadPart(boolean isLastPart) throws IOException {
126126
if (buffer.size() == 0) {
127127
return;
128128
}
@@ -137,7 +137,12 @@ private void uploadPart() throws IOException {
137137
}
138138
T partETag =
139139
multiPartUploadStore.uploadPart(
140-
objectName, uploadId, uploadedParts.size() + 1, tempFile, data.length);
140+
objectName,
141+
uploadId,
142+
uploadedParts.size() + 1,
143+
isLastPart,
144+
tempFile,
145+
data.length);
141146
uploadedParts.add(partETag);
142147
buffer.reset();
143148
} catch (Exception e) {

‎paimon-filesystems/paimon-azure/src/main/java/org/apache/paimon/azure/AzureLoader.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,6 @@
2525
import org.apache.paimon.fs.PluginFileIO;
2626
import org.apache.paimon.plugin.PluginLoader;
2727

28-
import java.util.ArrayList;
29-
import java.util.List;
30-
3128
/** Azure Blob Storage {@link FileIOLoader}. */
3229
public class AzureLoader implements FileIOLoader {
3330

@@ -51,14 +48,6 @@ public String getScheme() {
5148
return "abfs";
5249
}
5350

54-
@Override
55-
public List<String[]> requiredOptions() {
56-
List<String[]> options = new ArrayList<>();
57-
options.add(new String[] {"azure.account-name", "fs.azure.account.name"});
58-
options.add(new String[] {"azure.account-key", "fs.azure.account.key"});
59-
return options;
60-
}
61-
6251
@Override
6352
public FileIO load(Path path) {
6453
return new AzurePluginFileIO();

‎paimon-filesystems/paimon-gs-impl/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,11 @@
205205
<groupId>org.slf4j</groupId>
206206
<artifactId>slf4j-reload4j</artifactId>
207207
</exclusion>
208+
<!-- Exclude jeysey-json because of incompatible license -->
209+
<exclusion>
210+
<groupId>com.github.pjfanning</groupId>
211+
<artifactId>jersey-json</artifactId>
212+
</exclusion>
208213
</exclusions>
209214
</dependency>
210215
<dependency>

‎paimon-filesystems/paimon-hadoop-shaded/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,11 @@
178178
<groupId>org.slf4j</groupId>
179179
<artifactId>slf4j-reload4j</artifactId>
180180
</exclusion>
181+
<!-- Exclude jeysey-json because of incompatible license -->
182+
<exclusion>
183+
<groupId>com.github.pjfanning</groupId>
184+
<artifactId>jersey-json</artifactId>
185+
</exclusion>
181186
</exclusions>
182187
</dependency>
183188
</dependencies>

‎paimon-filesystems/paimon-hadoop-uber/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,11 @@
120120
<groupId>org.slf4j</groupId>
121121
<artifactId>slf4j-log4j12</artifactId>
122122
</exclusion>
123+
<!-- Exclude jeysey-json because of incompatible license -->
124+
<exclusion>
125+
<groupId>com.github.pjfanning</groupId>
126+
<artifactId>jersey-json</artifactId>
127+
</exclusion>
123128
</exclusions>
124129
</dependency>
125130

‎paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSMultiPartUpload.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,12 @@ public CompleteMultipartUploadResult completeMultipartUpload(
6060

6161
@Override
6262
public PartETag uploadPart(
63-
String objectName, String uploadId, int partNumber, File file, long byteLength)
63+
String objectName,
64+
String uploadId,
65+
int partNumber,
66+
boolean isLastPart,
67+
File file,
68+
long byteLength)
6469
throws IOException {
6570
return store.uploadPart(file, objectName, uploadId, partNumber);
6671
}

‎paimon-filesystems/paimon-oss-impl/src/test/java/org/apache/paimon/oss/OssTwoPhaseOutputStreamTest.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,12 @@ public String startMultiPartUpload(String objectName) {
239239

240240
@Override
241241
public PartETag uploadPart(
242-
String objectName, String uploadId, int partNumber, File file, long byteLength)
242+
String objectName,
243+
String uploadId,
244+
int partNumber,
245+
boolean isLastPart,
246+
File file,
247+
long byteLength)
243248
throws IOException {
244249
uploadPartCalls++;
245250

‎paimon-filesystems/paimon-s3-impl/pom.xml‎

Lines changed: 35 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,14 @@
6565
<artifactId>commons-beanutils</artifactId>
6666
<version>${commons.beanutils.version}</version>
6767
</dependency>
68+
69+
<dependency>
70+
<groupId>software.amazon.awssdk</groupId>
71+
<artifactId>bom</artifactId>
72+
<version>${fs.s3.aws.version}</version>
73+
<type>pom</type>
74+
<scope>import</scope>
75+
</dependency>
6876
</dependencies>
6977
</dependencyManagement>
7078

@@ -206,34 +214,34 @@
206214
<groupId>org.slf4j</groupId>
207215
<artifactId>slf4j-reload4j</artifactId>
208216
</exclusion>
217+
<!-- Exclude jeysey-json because of incompatible license -->
218+
<exclusion>
219+
<groupId>com.github.pjfanning</groupId>
220+
<artifactId>jersey-json</artifactId>
221+
</exclusion>
209222
</exclusions>
210223
</dependency>
211224

212225
<!-- AWS dependencies (bundled) -->
213226
<dependency>
214-
<groupId>com.amazonaws</groupId>
215-
<artifactId>aws-java-sdk-core</artifactId>
216-
<version>${fs.s3.aws.version}</version>
227+
<groupId>software.amazon.awssdk</groupId>
228+
<artifactId>s3</artifactId>
217229
</dependency>
218230
<dependency>
219-
<groupId>com.amazonaws</groupId>
220-
<artifactId>aws-java-sdk-s3</artifactId>
221-
<version>${fs.s3.aws.version}</version>
231+
<groupId>software.amazon.awssdk</groupId>
232+
<artifactId>s3-transfer-manager</artifactId>
222233
</dependency>
223234
<dependency>
224-
<groupId>com.amazonaws</groupId>
225-
<artifactId>aws-java-sdk-kms</artifactId>
226-
<version>${fs.s3.aws.version}</version>
235+
<groupId>software.amazon.awssdk</groupId>
236+
<artifactId>kms</artifactId>
227237
</dependency>
228238
<dependency>
229-
<groupId>com.amazonaws</groupId>
230-
<artifactId>aws-java-sdk-dynamodb</artifactId>
231-
<version>${fs.s3.aws.version}</version>
239+
<groupId>software.amazon.awssdk</groupId>
240+
<artifactId>dynamodb</artifactId>
232241
</dependency>
233242
<dependency>
234-
<groupId>com.amazonaws</groupId>
235-
<artifactId>aws-java-sdk-sts</artifactId>
236-
<version>${fs.s3.aws.version}</version>
243+
<groupId>software.amazon.awssdk</groupId>
244+
<artifactId>sts</artifactId>
237245
</dependency>
238246

239247
<!-- Hadoop's s3 support classes (bundled) -->
@@ -243,8 +251,8 @@
243251
<version>${fs.hadoopshaded.version}</version>
244252
<exclusions>
245253
<exclusion>
246-
<groupId>com.amazonaws</groupId>
247-
<artifactId>aws-java-sdk-bundle</artifactId>
254+
<groupId>software.amazon.awssdk</groupId>
255+
<artifactId>bundle</artifactId>
248256
</exclusion>
249257
<exclusion>
250258
<groupId>ch.qos.reload4j</groupId>
@@ -369,13 +377,18 @@
369377
<exclude>META-INF/LICENSE.txt</exclude>
370378
</excludes>
371379
</filter>
380+
<!--
381+
analyticsaccelerator-s3 puts these files under the root of the JAR,
382+
which causes license scanning tools to complain.
383+
This is a transitive dependency of hadoop-aws. It uses Apache License 2.0,
384+
and we don't bundle its transitive dependencies.
385+
-->
372386
<filter>
373-
<artifact>com.amazonaws:aws-java-sdk-s3</artifact>
374-
<!-- Make sure we are using the overridden XmlResponsesSaxParser.
375-
Filter must be removed as soon as XmlResponsesSaxParser of this module is
376-
dropped, for example when discontinuing support for Java 8. -->
387+
<artifact>software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3</artifact>
377388
<excludes>
378-
<exclude>com/amazonaws/services/s3/model/transform/XmlResponsesSaxParser**</exclude>
389+
<exclude>LICENSE</exclude>
390+
<exclude>NOTICE</exclude>
391+
<exclude>THIRD-PARTY-NOTICES</exclude>
379392
</excludes>
380393
</filter>
381394
</filters>

0 commit comments

Comments
 (0)