Skip to content

Commit 6760a02

Browse files
Zhangxunmtsimonelbaz
Zhangxunmt
authored and
simonelbaz
committed
create s3 common module (opensearch-project#6404)
Signed-off-by: Xun Zhang <xunzh@amazon.com>
1 parent 7b41fe8 commit 6760a02

54 files changed

Lines changed: 290 additions & 92 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎data-prepper-plugins/s3-common/build.gradle‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
dependencies {
12+
compileOnly 'org.projectlombok:lombok:1.18.30'
13+
annotationProcessor 'org.projectlombok:lombok:1.18.30'
14+
implementation project(path: ':data-prepper-plugins:common')
15+
implementation 'software.amazon.awssdk:sts'
16+
implementation 'software.amazon.awssdk:s3'
17+
implementation 'com.fasterxml.jackson.core:jackson-databind'
18+
implementation 'org.apache.httpcomponents:httpcore:4.4.16'
19+
implementation 'dev.failsafe:failsafe:3.3.2'
20+
implementation 'io.micrometer:micrometer-core'
21+
testImplementation libs.commons.lang3
22+
implementation libs.parquet.common
23+
testImplementation testLibs.bundles.junit
24+
}
25+
26+
jacocoTestCoverageVerification {
27+
dependsOn jacocoTestReport
28+
violationRules {
29+
rule { //in addition to core projects rule
30+
limit {
31+
minimum = 0.90
32+
}
33+
}
34+
}
35+
}

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/configuration/AwsAuthenticationOptions.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/config/AwsAuthenticationOptions.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
49
*/
510

6-
package org.opensearch.dataprepper.plugins.source.s3.configuration;
11+
package org.opensearch.dataprepper.plugins.s3.common.config;
712

813
import com.fasterxml.jackson.annotation.JsonProperty;
914
import jakarta.validation.constraints.Size;
10-
import software.amazon.awssdk.arns.Arn;
1115
import software.amazon.awssdk.regions.Region;
16+
import software.amazon.awssdk.arns.Arn;
1217

1318
import java.util.Map;
1419
import java.util.Optional;
@@ -68,4 +73,3 @@ public Map<String, String> getAwsStsHeaderOverrides() {
6873
return awsStsHeaderOverrides;
6974
}
7075
}
71-

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/BucketOwnerProvider.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/BucketOwnerProvider.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
49
*/
510

6-
package org.opensearch.dataprepper.plugins.source.s3.ownership;
11+
package org.opensearch.dataprepper.plugins.s3.common.ownership;
712

813
import java.util.Optional;
914

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/MappedBucketOwnerProvider.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/MappedBucketOwnerProvider.java

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,30 @@
1-
package org.opensearch.dataprepper.plugins.source.s3.ownership;
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
package org.opensearch.dataprepper.plugins.s3.common.ownership;
212

313
import java.util.HashMap;
414
import java.util.Map;
515
import java.util.Objects;
616
import java.util.Optional;
7-
817
/**
918
* Implements {@link BucketOwnerProvider} using a mapping of bucket
1019
* names to account Ids for the bucket owners. Uses a delegate
1120
* {@link BucketOwnerProvider} as a fallback when the bucket is not
1221
* found in the map.
1322
*/
14-
class MappedBucketOwnerProvider implements BucketOwnerProvider {
23+
public class MappedBucketOwnerProvider implements BucketOwnerProvider {
1524
private final Map<String, String> bucketOwnershipMap;
1625
private final BucketOwnerProvider fallbackProvider;
1726

18-
MappedBucketOwnerProvider(Map<String, String> bucketOwnershipMap, BucketOwnerProvider fallbackProvider) {
27+
public MappedBucketOwnerProvider(Map<String, String> bucketOwnershipMap, BucketOwnerProvider fallbackProvider) {
1928
this.bucketOwnershipMap = new HashMap<>(Objects.requireNonNull(bucketOwnershipMap));
2029
this.fallbackProvider = Objects.requireNonNull(fallbackProvider);
2130
}

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/NoOwnershipBucketOwnerProvider.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/NoOwnershipBucketOwnerProvider.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
49
*/
510

6-
package org.opensearch.dataprepper.plugins.source.s3.ownership;
11+
package org.opensearch.dataprepper.plugins.s3.common.ownership;
712

813
import java.util.Optional;
914

1015
/**
1116
* An implementation of {@link BucketOwnerProvider} which does not provide
1217
* a bucket owner, effectively skipping owner validation.
1318
*/
14-
class NoOwnershipBucketOwnerProvider implements BucketOwnerProvider {
19+
public class NoOwnershipBucketOwnerProvider implements BucketOwnerProvider{
1520
@Override
1621
public Optional<String> getBucketOwner(final String bucket) {
1722
return Optional.empty();

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ownership/StaticBucketOwnerProvider.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/ownership/StaticBucketOwnerProvider.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,19 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
49
*/
510

6-
package org.opensearch.dataprepper.plugins.source.s3.ownership;
11+
package org.opensearch.dataprepper.plugins.s3.common.ownership;
712

813
import java.util.Objects;
914
import java.util.Optional;
1015

11-
/**
12-
* An implementation of {@link BucketOwnerProvider} which provides the
13-
* same owner for all buckets.
14-
*/
15-
class StaticBucketOwnerProvider implements BucketOwnerProvider {
16+
public class StaticBucketOwnerProvider implements BucketOwnerProvider {
1617
private final String accountId;
1718

1819
public StaticBucketOwnerProvider(final String accountId) {

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputFile.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputFile.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,18 @@
1-
package org.opensearch.dataprepper.plugins.source.s3;
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
package org.opensearch.dataprepper.plugins.s3.common.source;
212

313
import org.apache.parquet.io.SeekableInputStream;
414
import org.opensearch.dataprepper.model.io.InputFile;
5-
import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider;
15+
import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider;
616
import software.amazon.awssdk.services.s3.S3Client;
717
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
818
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3InputStream.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3InputStream.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,14 @@
1-
package org.opensearch.dataprepper.plugins.source.s3;
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
package org.opensearch.dataprepper.plugins.s3.common.source;
212

313
import com.google.common.base.Preconditions;
414
import com.google.common.io.ByteStreams;
@@ -8,7 +18,7 @@
818
import dev.failsafe.function.CheckedSupplier;
919
import org.apache.http.ConnectionClosedException;
1020
import org.apache.parquet.io.SeekableInputStream;
11-
import org.opensearch.dataprepper.plugins.source.s3.ownership.BucketOwnerProvider;
21+
import org.opensearch.dataprepper.plugins.s3.common.ownership.BucketOwnerProvider;
1222
import org.slf4j.Logger;
1323
import org.slf4j.LoggerFactory;
1424
import software.amazon.awssdk.core.sync.ResponseTransformer;

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectPluginMetrics.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectPluginMetrics.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
49
*/
5-
package org.opensearch.dataprepper.plugins.source.s3;
10+
11+
package org.opensearch.dataprepper.plugins.s3.common.source;
612

713
import io.micrometer.core.instrument.Counter;
814
import io.micrometer.core.instrument.DistributionSummary;
@@ -94,4 +100,4 @@ public Counter getS3ObjectsThrottledCounter() {
94100
public Counter getS3ObjectsDeleteFailed() { return s3ObjectsDeleteFailed; }
95101

96102
public Counter getS3ObjectReadFailedCounter() { return s3ObjectReadFailedCounter; }
97-
}
103+
}

data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ObjectReference.java renamed to data-prepper-plugins/s3-common/src/main/java/org/opensearch/dataprepper/plugins/s3/common/source/S3ObjectReference.java

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,27 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
49
*/
510

6-
package org.opensearch.dataprepper.plugins.source.s3;
11+
package org.opensearch.dataprepper.plugins.s3.common.source;
12+
13+
import lombok.Getter;
714

815
import java.util.Objects;
916
import java.util.Optional;
1017

1118
/**
1219
* Reference to an S3 object.
1320
*/
14-
class S3ObjectReference {
21+
public class S3ObjectReference {
22+
@Getter
1523
private final String bucketName;
24+
@Getter
1625
private final String key;
1726
private final String owner;
1827

@@ -22,20 +31,12 @@ private S3ObjectReference(final String bucketName, final String key, final Strin
2231
this.owner = owner;
2332
}
2433

25-
static Builder bucketAndKey(final String bucketName, final String key) {
34+
public static Builder bucketAndKey(final String bucketName, final String key) {
2635
Objects.requireNonNull(bucketName, "bucketName must be non null");
2736
Objects.requireNonNull(key, "key must be non null");
2837
return new Builder(bucketName, key);
2938
}
3039

31-
String getBucketName() {
32-
return bucketName;
33-
}
34-
35-
String getKey() {
36-
return key;
37-
}
38-
3940
Optional<String> getBucketOwner() {
4041
return Optional.ofNullable(owner);
4142
}

0 commit comments

Comments
 (0)